import threading
from copy import copy

import requests
from DrissionPage._base.chromium import Chromium
from DrissionPage._configs.chromium_options import ChromiumOptions
from concurrent.futures import ThreadPoolExecutor, as_completed, wait

from DrissionPage._functions.settings import Settings
from DrissionPage._pages.session_page import SessionPage
from DrissionPage.errors import ElementNotFoundError, WaitTimeoutError, PageDisconnectedError
from fake_useragent import UserAgent
from requests.adapters import HTTPAdapter
from sqlalchemy import func, text
from sqlmodel import select
from urllib3 import Retry

from config.db import get_session
from constant.crawler_constant import HouseDetailKey, JiNanUrlId
from crawler_handler.abstract_crawler_handler import AbstractCrawlerHandler
from decorators.exception_decorator import crawler_exception_handler
from dict.crwaler_dict import house_state_dict, project_base_info_dict
from entity.model.community_model import CommunityDataCollection
from entity.model.crawler_record_model import CommunityProjectRecord, CommunityBuildingRecord
from entity.schema.crawler_schema import CrawlerArgs
from enums.crawler_enum import ProjectType, HouseState
from util.crawler_util import get_query_params, random_sleep_thread
from util.data_change_util import change_to_float, change_to_int, get_date_str, get_digit
from util.mylog import my_logger

executor = ThreadPoolExecutor(max_workers=10, thread_name_prefix='com_pro')
data_executor = ThreadPoolExecutor(max_workers=30, thread_name_prefix='data_detail')
community_executor = ThreadPoolExecutor(max_workers=5, thread_name_prefix='community')
adapter = HTTPAdapter(pool_connections=50, pool_maxsize=100, max_retries=Retry(total=5, backoff_factor=5, status_forcelist=[500, 502, 503, 504]))
thread_local = threading.local()

class JinanHandler(AbstractCrawlerHandler):

    community_url = 'https://jncc.jinan.gov.cn/jnfdcinfo/jnfdcweb/onsaling/index_{}.shtml?zn={}&pu={}&pn={}&en={}'
    community_project_url_prefix = 'https://jncc.jinan.gov.cn/jnfdcinfo/jnfdcweb/onsaling/show_{}.shtml?prjno={}'
    build_table_url_prefix = 'https://jncc.jinan.gov.cn/jnfdcinfo/jnfdcweb/onsaling/viewhouse.shtml?fmid={}'
    build_info_url_prefix = 'https://jncc.jinan.gov.cn/jnfdcinfo/jnfdcweb/onsaling/bshow.shtml?bno={}'
    room_link_prefix = 'https://jncc.jinan.gov.cn/jnfdcinfo/jnfdcweb/onsaling/viewDiv.shtml?fid={}'

    def __init__(self, browser: Chromium, crawler_args: CrawlerArgs):
        super().__init__(browser)
        self.crawler_args = crawler_args
        self.all_counts = None
        self.all_page = None

    # 获取楼盘表里住宅的信息
    def get_house_info(self, model: CommunityDataCollection):
        try:
            link = JinanHandler.room_link_prefix.format(model.room_id)
            my_logger.info(f"爬取楼盘表信息:{link}")
            data = self._create_session().get(link, timeout=20).json()
            # my_logger.info(type(data))
            my_logger.info(data)
            model.building_area = data.get(HouseDetailKey.BUILDING_AREA, None)
            # todo 待转换
            model.inner_area = data.get(HouseDetailKey.INNER_AREA, None)
            model.public_area = data.get(HouseDetailKey.PUBLIC_AREA, None)
            model.purpose = data.get(HouseDetailKey.PURPOSE, None)
            #
            house_state = data.get(HouseDetailKey.SALE_STATUS, None)
            if house_state:
                house_state = change_to_int(house_state)
                model.sale_status = house_state_dict[HouseState._value2member_map_[house_state]]
            # 备案时间
            synchrotime = data.get(HouseDetailKey.FILING_TIME, None)
            if synchrotime:
                model.filing_time = get_date_str(synchrotime)
            model.data_source = 1
        except Exception as e:
            my_logger.error(f"Error get_house_info task: {e}")
        # my_logger.warning("dddd")
        # return data

    def process_community_table(self, model: CommunityDataCollection):
        with self.lock:
            tab = self.browser.new_tab()
        try:
            lnk = JinanHandler.build_table_url_prefix.format(model.building_id)
            # my_logger.info(f"爬取楼盘表链接:{lnk}")
            i = 0
            while True:
                try:
                    tab.get(lnk)
                    tab.wait(1)
                    data_table = tab.ele('#floorTable')
                    break
                except ElementNotFoundError as e:
                    my_logger.warning(f"Error processing house info task: {e}")
                    my_logger.warning(f"可能是被限制了, 等待60秒后重新请求: link: {lnk}")
                    tab.wait(60)
                    i += 1
                    if i > 3:
                        return False, model
                except WaitTimeoutError as e:
                    my_logger.warning(e)
                    my_logger.warning(f"失败 link: {lnk}")
                    # my_logger.warning("重试")
                    tab.wait(30)
                    i += 1
                    if i > 3:
                        return False, model
            tab.wait(1)
            tr_list = data_table.s_eles('@tag()=tr')
            # 单元映射表
            unit_map = {}
            floor = None
            room_model_list = []
            for idx_row, tr in enumerate(tr_list):
                td_list = tr.children()
                for idx_td, td in enumerate(td_list):
                    # 单元
                    if idx_row == 0 and idx_td > 0:
                        model.unit_number = JinanHandler.convert_unit(td.text)
                        model.max_room_no = JinanHandler.convert_total_unit(td.attr('colspan'))
                        unit_model = copy(model)
                        # 当前单元所占房号
                        l = len(unit_map)
                        for idx in range(model.max_room_no):
                            unit_map[idx + l] = unit_model
                    # 室号
                    # 户
                    if idx_row >= 2:
                        # 楼层
                        if idx_td == 0:
                            floor = JinanHandler.convert_floor(td.text)
                            # if idx_row == 2:
                            # 最大楼层
                            # my_logger.info(f"最大楼层： {floor}")
                        # id
                        if idx_td > 0:
                            # 房间号
                            room_model = copy(unit_map[idx_td - 1])
                            room_data_id = td.attr('id')
                            if room_data_id:
                                room_model.room_id = room_data_id
                                # todo 待处理
                                room_model.room_number = td.text
                                room_model.floor = floor
                                # my_logger.info(room_model)
                                room_model_list.append(room_model)
            tab.close()
            # if not room_model_list:
            #     my_logger.error(f"未获取到房间信息: {model}")
            #     my_logger.error(lnk)
            return True, room_model_list
        except Exception as e:
            my_logger.error(f"tab: {tab} process_community_table: {e}")
            return False, model

    @crawler_exception_handler
    def get_community_building_link(self, page_num: int, community_id):
        community_link = JinanHandler.community_project_url_prefix.format(page_num, community_id)
        page = JinanHandler.fetch_page(community_link)
        community_building_list = page.eles('.list-main')
        page.close()
        return True, community_building_list
    def get_community_building_link_by_tab(self, page_num: int, community_id):
        with self.lock:
            tab = self.browser.new_tab()
        while True:
            try:
                community_link = JinanHandler.community_project_url_prefix.format(page_num, community_id)
                tab.get(community_link)
                community_building_list = tab.eles('.list-main')
                break
            except ElementNotFoundError as e:
                my_logger.warning(f"Error get_community_building_link: {e} / {community_id}")
                random_sleep_thread("get_community_building_link", 5, 10)
            except Exception as e:
                my_logger.error(f"Error get_community_building_link: {e} / {community_id}")
                return []
        # tab.close()
        return community_building_list

    def collect_community_building(self, all_page: int, community_id):
        futures = []
        for page_num in range(all_page):
            my_logger.info(f"正在获取第{page_num + 1}页")
            futures.append(community_executor.submit(self.get_community_building_link, page_num + 1, community_id))
        JinanHandler.process_futures(futures)
        return JinanHandler.process_futures(futures)

    def process_community_building_info_link(self, model: CommunityDataCollection):
        try:
            community_link = JinanHandler.community_project_url_prefix.format(1, model.community_id)
            # my_logger.info(f"楼盘链接:{community_link}")
            community_building_page = JinanHandler.fetch_page(community_link)
            all_counts = community_building_page.ele('#allcounts').attr('value')
            all_page = community_building_page.ele('#allpage').attr('value')
            my_logger.info(f"当前楼栋总数量:{all_counts}, 总页数:{all_page}")
            # 楼盘基本信息
            community_detail_info_list = community_building_page.s_eles('.massage_bg')
            for info in community_detail_info_list:
                text = info.text
                base_text = info.next().text
                if text in project_base_info_dict:
                    setattr(model, project_base_info_dict[text], base_text)
            # 每栋的楼栋信息
            community_building_list = self.collect_community_building(int(all_page), model.community_id)
            community_building_model_list = []
            community_building_page.close()
            for item in community_building_list:
                # 楼盘表链接
                community_table = item.ele('#houseshow')
                if not community_table or not community_table.link:
                    continue
                community_table_link = community_table.link
                # Shallow copy
                community_building_model = copy(model)
                # 楼栋id同楼盘表id
                community_building_id = JinanHandler.get_community_table_id(community_table_link)
                community_building_model.building_id = community_building_id
                # 获取楼盘基本信息
                page = JinanHandler.fetch_page(JinanHandler.build_info_url_prefix.format(community_building_id))
                building_info_table = page.ele('.message_table')
                tr_list = building_info_table.s_eles('@tag()=tr')
                # 楼栋名称
                tr_build_name = tr_list[1].s_ele('#lblFMonoName').text
                # 许可证号
                tr_certificate_no = tr_list[-1].s_eles('.massage_bg')[1].text
                # 监管银行和监管账号
                band_account = tr_list[-2].s_eles('.massage_bg')
                tr_bank = band_account[1].text
                tr_account = band_account[-1].text
                community_building_model.building_number = tr_build_name
                community_building_model.certificate_no = tr_certificate_no
                community_building_model.regulatory_banks = tr_bank
                community_building_model.regulatory_account = tr_account
                community_building_model_list.append(community_building_model)
                page.close()
            return True, community_building_model_list
        except Exception as e:
            my_logger.error(f"Error process_community_building_info_link task: {e}")
            return False, model

    def get_community_building_info(self, community_project_model: CommunityProjectRecord):
        model = CommunityBuildingRecord(**community_project_model.model_dump(exclude={"id"}))
        community_link = JinanHandler.community_project_url_prefix.format(1, model.community_id)
        try:
            # my_logger.info(f"楼盘链接:{community_link}")
            community_building_page = JinanHandler.fetch_page(community_link)
            # todo 可能没房子
            all_counts = community_building_page.ele('#allcounts').attr('value')
            all_page = community_building_page.ele('#allpage').attr('value')
            my_logger.info(f"当前楼栋总数量:{all_counts}, 总页数:{all_page}")
            # 每栋的楼栋信息
            community_building_list, fail_community_id_list = self.collect_community_building(int(all_page), model.community_id)
            # 楼盘基本信息
            community_detail_info_list = community_building_page.s_eles('.massage_bg')
            for info in community_detail_info_list:
                text = info.text
                base_text = info.next().text
                if text in project_base_info_dict:
                    setattr(model, project_base_info_dict[text], base_text)
            community_building_model_list = []
            community_building_page.close()
            for item in community_building_list:
                # 楼盘表链接
                community_table = item.ele('#houseshow')
                if not community_table or not community_table.link:
                    continue
                community_table_link = community_table.link
                # Shallow copy
                community_building_model = copy(model)
                # 楼栋id同楼盘表id
                community_building_id = JinanHandler.get_community_table_id(community_table_link)
                community_building_model.building_id = community_building_id
                # 获取楼盘基本信息
                page = JinanHandler.fetch_page(JinanHandler.build_info_url_prefix.format(community_building_id))
                building_info_table = page.ele('.message_table')
                tr_list = building_info_table.s_eles('@tag()=tr')
                # 楼栋名称
                tr_build_name = tr_list[1].s_ele('#lblFMonoName').text
                # 许可证号
                tr_certificate_no = tr_list[-1].s_eles('.massage_bg')[1].text
                # 监管银行和监管账号
                band_account = tr_list[-2].s_eles('.massage_bg')
                community_building_model.building_number = tr_build_name
                community_building_model.certificate_no = tr_certificate_no
                tr_bank = band_account[1].text
                tr_account = band_account[-1].text
                community_building_model.regulatory_banks = tr_bank
                community_building_model.regulatory_account = tr_account
                # 是否是住宅
                form_list_main_list = page.eles('.list-main')
                if form_list_main_list:
                    is_house_flag = False
                    for form_list_main in form_list_main_list:
                        form_ul = form_list_main.child()
                        li_text_list = form_ul.s_eles('.ellipsis')
                        community_building_model.sale_count = change_to_int(li_text_list[-4].text)
                        community_building_model.can_sale_count = change_to_int(li_text_list[-2].text)
                        if li_text_list[0].text == '住宅':
                            community_building_model.is_house = '0'
                            is_house_flag = True
                            break
                    if not is_house_flag:
                        community_building_model.is_house = '1'
                else:
                    # 啥房都没有
                    community_building_model.is_house = '0'
                    community_building_model.can_sale_count = 0
                    community_building_model.sale_count = 0
                community_building_model_list.append(community_building_model)
                page.close()
            return True, community_building_model_list
        except ElementNotFoundError as e:
            my_logger.warning(f"元素未找到: {e}")
            my_logger.warning(f"链接：{community_link}")
            model.building_id = None
            return False, model
        except Exception as e:
            my_logger.error(type(e))
            my_logger.error(f"Error process_community_building_info_link task: {e}")
            model.building_id = None
            return False, model

    @crawler_exception_handler
    def get_community_link(self, page_num):
        link = JinanHandler.community_url.format(page_num + 1, 'all', self.crawler_args.project_type.value, '', '')
        page = JinanHandler.fetch_page(link)
        community_links = page.s_ele('.form-list').s_eles('t:a')
        page.close()
        return True, community_links

    def collect_community_project(self):
        futures = []
        for page_num in range(self.all_page):
            my_logger.info(f"正在获取第{page_num + 1}页")
            futures.append(executor.submit(self.get_community_link, page_num))
        success_task, fail_task = JinanHandler.process_futures(futures)
        if fail_task:
           my_logger.error(f"获取楼盘链接失败:{fail_task}")
        return success_task

    # 获取楼盘表里住宅的信息
    def collect_house_info(self, model: CommunityDataCollection):
        try:
            link = JinanHandler.room_link_prefix.format(model.room_id)
            my_logger.info(f"爬取楼盘表信息:{link}")
            data = self._create_session().get(link, timeout=20).json()
            # my_logger.info(type(data))
            my_logger.info(data)
            model.building_area = change_to_float(data.get(HouseDetailKey.BUILDING_AREA, None))
            model.inner_area = change_to_float(data.get(HouseDetailKey.INNER_AREA, None))
            model.public_area = change_to_float(data.get(HouseDetailKey.PUBLIC_AREA, None))
            model.purpose = data.get(HouseDetailKey.PURPOSE, None)
            #
            house_state = data.get(HouseDetailKey.SALE_STATUS, None)
            if house_state:
                house_state = change_to_int(house_state)
                model.sale_status = house_state_dict[HouseState._value2member_map_[house_state]]
            # 备案时间
            synchrotime = data.get(HouseDetailKey.FILING_TIME, None)
            if synchrotime:
                model.filing_time = get_date_str(synchrotime)
            model.data_source = 1
        except Exception as e:
            my_logger.error(f"Error get_house_info task: {e}")
        # my_logger.warning("dddd")
        # return data

    # 获取楼盘表和房间基本信息(floor, room_number, unit)
    def collect_community_table_and_room_info(self, community_building_model: CommunityBuildingRecord):
        model = CommunityDataCollection(**community_building_model.model_dump(exclude={"id"}))
        with self.lock:
            tab = self.browser.new_tab()
        lnk = JinanHandler.build_table_url_prefix.format(model.building_id)
        i = 0
        while True:
            try:
                tab.get(lnk)
                body = tab.s_ele('@tag()=body', timeout=15)
                if not body:
                    my_logger.info("页面为空!")
                    return False, model
                tab.wait(1)
                data_table = body.ele('#floorTable', timeout=30)
                tab.wait(1)
                tr_list = data_table.s_eles('@tag()=tr')
                # 单元映射表
                unit_map = {}
                floor = None
                room_model_list = []
                # 房间楼层映射表
                room_floor_map = {}
                for idx_row, tr in enumerate(tr_list):
                    td_list = tr.children()
                    for idx_td, td in enumerate(td_list):
                        # 单元
                        if idx_row == 0 and idx_td > 0:
                            model.unit_number = JinanHandler.convert_unit(td.text)
                            model.max_room_no = JinanHandler.convert_total_unit(td.attr('colspan'))
                            unit_model = copy(model)
                            # 当前单元所占房号
                            l = len(unit_map)
                            for idx in range(model.max_room_no):
                                unit_map[idx + l] = unit_model
                        # 室号
                        # 户
                        if idx_row >= 2:
                            # 楼层
                            if idx_td == 0:
                                floor = JinanHandler.convert_floor(td.text)
                                # if idx_row == 2:
                                # 最大楼层
                                # my_logger.info(f"最大楼层： {floor}")
                            # id
                            if idx_td > 0:
                                # 房间号
                                room_model = copy(unit_map[idx_td - 1])
                                room_data_id = td.attr('id')
                                if room_data_id:
                                    room_model.room_id = room_data_id
                                    room_model.room_number = td.text
                                    # room_model.room_number = JinanHandler.convert_room_number(td.text)
                                    # 判断房间是否掉下来
                                    current_room_id_value = get_digit(room_data_id)
                                    if current_room_id_value in room_floor_map:
                                        room_model.floor = room_floor_map[current_room_id_value]
                                    else:
                                        room_model.floor = floor
                                    # my_logger.info(room_model)
                                    room_model_list.append(room_model)
                                    # 加入表中
                                    room_floor_map[current_room_id_value] = room_model.floor
                                else:
                                    if not room_model_list:
                                        continue
                                    # 拿到前面一个房间的数据(Python 3.7 字典有序)
                                    front_room_id_value = list(room_floor_map.keys())[-1]
                                    # 当前位置的id应是前一个房间的id的值 + 1
                                    cur_room_id_value = front_room_id_value + 1
                                    # 也把当前没有房间的id加入表中
                                    room_floor_map[cur_room_id_value] = floor
                tab.close()
                # if not room_model_list:
                #     my_logger.error(f"未获取到房间信息: {model}")
                #     my_logger.error(lnk)
                return True, room_model_list
            except ElementNotFoundError as e:
                my_logger.warning(f"Error processing house info task: {e}")
                my_logger.warning(f"可能是被限制了, 等待60秒后重新请求: link: {lnk}")
                tab.wait(60)
                i += 1
                if i > 3:
                    return False, model
            except PageDisconnectedError as e:
                my_logger.warning(f"Error processing house info task: {e}")
                my_logger.warning(f"可能网不好, 等待30秒后重新请求: link: {lnk}")
                tab.wait(30)
                i += 1
                if i > 3:
                    return False, model
            except WaitTimeoutError as e:
                my_logger.warning(e)
                my_logger.warning(f"失败 link: {lnk}")
                # my_logger.warning("重试")
                tab.wait(30)
                i += 1
                if i > 3:
                    return False, model
            except Exception as e:
                my_logger.error(f"data_table: {data_table} coll_community_table: {e}, link: {lnk}")
                model.room_id = None
                return False, model



    def crawler_community(self):
        link = JinanHandler.community_url.format(1, 'all', self.crawler_args.project_type.value, '', '')
        page = JinanHandler.fetch_page(link)
        all_counts = page.ele('#allcounts').attr('value')
        all_page = page.ele('#allpage').attr('value')
        my_logger.info(f"总数量:{all_counts}, 总页数:{all_page}")
        self.all_counts = int(all_counts)
        self.all_page = int(all_page)
        page.close()
        community_project_links = self.collect_community_project()
        # community_project_links = page.s_ele('.form-list').s_eles('t:a')
        community_link_task = len(community_project_links)
        my_logger.info(f"获取到链接数量:{community_link_task}")
        # 楼栋链接
        futures = []
        for lnk in community_project_links:
            link = lnk.link
            if not link or not link.startswith('https'):
                continue
            community_model = CommunityDataCollection()
            community_id = JinanHandler.get_community_link_id(link)
            community_model.province = "山东省"
            community_model.province_id = 37
            community_model.area = "济南市"
            community_model.area_id = 3701
            community_model.community_id = community_id
            futures.append(executor.submit(self.process_community_building_info_link, community_model))
        fail_building_model_list = []
        community_table_futures = []
        for idx, future in enumerate(as_completed(futures)):
            try:
                # my_logger.info(future.result())
                is_success, result = future.result()
                my_logger.info(f"处理第{idx + 1} / {community_link_task}个任务完成")
                if is_success:
                    for model in result:
                        community_table_futures.append(community_executor.submit(self.process_community_table, model))
                else:
                    fail_building_model_list.append(result)
            except Exception as e:
                my_logger.error(f"Error processing community link task: {e}")
        # my_logger.info(f"ctf: {len(community_data_col_building_model_list)}")
        my_logger.warning(f"失败获取楼栋任务数量: {len(fail_building_model_list)}")
        if fail_building_model_list:
            JinanHandler.bulk_insert(CommunityDataCollection, fail_building_model_list)
        task_cnt = len(community_table_futures)
        # 等待所有任务完成
        fail_task = []
        for idx, community_table_future in enumerate(as_completed(community_table_futures)):
            try:
                is_success, data = community_table_future.result()
                my_logger.info(f"ct {idx + 1}, all: {task_cnt}.")
                if is_success:
                    house_info_futures = []
                    for room_model in data:
                        house_info_futures.append(data_executor.submit(self.get_house_info, room_model))
                    wait(house_info_futures)
                    JinanHandler.bulk_insert(CommunityDataCollection, data)
                else:
                    fail_task.append(data)
                    my_logger.error(f"获取楼栋数据失败：{data}")
            except Exception as e:
                my_logger.error(f"Error processing community table task: {e}")
        # 重试失败的task
        # if fail_task:
        #     self.browser.quit()
        #     cof = ChromiumOptions().auto_port()
        #     self.browser = Chromium(cof)
        fail_future_list = []
        for model in fail_task:
            fail_future_list.append(community_executor.submit(self.process_community_table, model))
        task_cnt = len(fail_future_list)
        re_fail_task_list = []
        for idx, community_table_future in enumerate(as_completed(fail_future_list)):
            try:
                is_success, data = community_table_future.result()
                my_logger.info(f"ct {idx + 1}, all: {task_cnt}.")
                if is_success:
                    house_info_futures = []
                    for room_model in data:
                        house_info_futures.append(data_executor.submit(self.get_house_info, room_model))
                    wait(house_info_futures)
                    JinanHandler.bulk_insert(CommunityDataCollection, data)
                else:
                    re_fail_task_list.append(data)
                    my_logger.error(f"获取楼栋数据失败：{data}")
                if re_fail_task_list:
                    JinanHandler.bulk_insert(CommunityDataCollection, re_fail_task_list)
            except Exception as e:
                my_logger.error(f"Error processing community table task: {e}")
        my_logger.info("crawler_community success")

    # 爬项目
    def crawler_project(self):
        link = JinanHandler.community_url.format(1, 'all', self.crawler_args.project_type.value, '', '')
        page = JinanHandler.fetch_page(link)
        all_counts = page.ele('#allcounts').attr('value')
        all_page = page.ele('#allpage').attr('value')
        my_logger.info(f"总数量:{all_counts}, 总页数:{all_page}")
        # 判断楼盘是否新增
        session = next(get_session())
        with session:
            stat = select(func.count()).select_from(CommunityProjectRecord)
            count = session.exec(stat).one()
        self.all_counts = int(all_counts)
        if count == self.all_counts:
            my_logger.info("楼盘数据未更新")
            return
        self.all_page = int(all_page)
        page.close()
        # todo如何判断新增
        community_project_links = self.collect_community_project()
        # community_project_links = page.s_ele('.form-list').s_eles('t:a')
        community_link_task = len(community_project_links)
        my_logger.info(f"获取到链接数量:{community_link_task}")
        # 楼栋链接
        community_project_model_list = []
        for lnk in community_project_links:
            link = lnk.link
            if not link or not link.startswith('https'):
                continue
            community_model =  CommunityProjectRecord()
            community_id = JinanHandler.get_community_link_id(link)
            community_model.province = "山东省"
            community_model.province_id = 37
            community_model.area = "济南市"
            community_model.area_id = 3701
            community_model.community_id = community_id
            community_project_model_list.append(community_model)
        JinanHandler.bulk_insert(CommunityProjectRecord, community_project_model_list)

    # 爬楼栋
    def crawler_building(self):
        session = next(get_session())
        with session:
            statement = select(CommunityProjectRecord)
            community_project_link_list = session.exec(statement).all()
        community_link_task = len(community_project_link_list)
        my_logger.info(f"楼栋数量:{community_link_task}")
        # 楼栋
        futures = []
        for community_model in community_project_link_list:
            futures.append(executor.submit(self.get_community_building_info, community_model))
        fail_building_model_list = []
        success_task, fail_task = JinanHandler.process_futures(futures)
        JinanHandler.bulk_insert(CommunityBuildingRecord, success_task)
        my_logger.warning(f"失败获取楼栋任务数量: {len(fail_task)}")
        if fail_task:
            my_logger.warning(f"失败获取楼栋任务: {fail_task}")
            # 失败的 building_id = None
            # JinanHandler.bulk_insert(CommunityBuildingRecord, fail_task)

    # 爬房间
    def crawler_room(self):
        session = next(get_session())
        with (session):
            statement = select(CommunityBuildingRecord.building_id, CommunityBuildingRecord.sale_count, CommunityBuildingRecord.can_sale_count
                               ).where(CommunityBuildingRecord.is_house == '0').group_by(CommunityBuildingRecord.building_id, CommunityBuildingRecord.sale_count, CommunityBuildingRecord.can_sale_count)
            building_id_room_count_list = session.exec(statement).all()
            futures = []
            for building_id_room_count in building_id_room_count_list:
                building_id, sale_count, can_sale_count = building_id_room_count
                stat = select(func.count()).select_from(CommunityDataCollection).where(CommunityDataCollection.building_id == building_id)
                count = session.exec(stat).one()
                # 房间数小于售房数+可售房数，则爬取该楼栋数据
                if count < sale_count + can_sale_count:
                    stat = select(CommunityBuildingRecord).where(CommunityBuildingRecord.building_id == building_id)
                    building_data = session.exec(stat).one()
                    futures.append(executor.submit(self.collect_community_table_and_room_info, building_data))
            success_task, fail_task = JinanHandler.process_futures(futures)
            JinanHandler.bulk_insert(CommunityDataCollection, success_task)
            # todo 失败任务处理
            my_logger.warning(f"失败获取房间任务数量: {len(fail_task)}")
            if fail_task:
                my_logger.warning(f"失败获取房间任务: {fail_task}")
    # 爬房间详情
    def crawler_room_detail(self):
        session = next(get_session())
        with session:
            stat = select(CommunityDataCollection.building_id).group_by(CommunityDataCollection.building_id)
            building_id_list = session.exec(stat).all()
            for building_id in building_id_list:
                stat = select(CommunityDataCollection).where(CommunityDataCollection.building_id == building_id)
                building_data_list = session.exec(stat).all()
                futures = []
                for building_data in building_data_list:
                    futures.append(data_executor.submit(self.collect_house_info, building_data))
                wait(futures)
                JinanHandler.bulk_update(CommunityDataCollection, building_data_list)

    def get_community_building_info_by_building_model(self, community_building_model: CommunityBuildingRecord):
        # 获取楼盘基本信息
        page = JinanHandler.fetch_page(JinanHandler.build_info_url_prefix.format(community_building_model.community_building_id))
        building_info_table = page.ele('.message_table')
        tr_list = building_info_table.s_eles('@tag()=tr')
        # 楼栋名称
        tr_build_name = tr_list[1].s_ele('#lblFMonoName').text
        # 许可证号
        tr_certificate_no = tr_list[-1].s_eles('.massage_bg')[1].text
        # 监管银行和监管账号
        band_account = tr_list[-2].s_eles('.massage_bg')
        tr_bank = band_account[1].text
        tr_account = band_account[-1].text
        community_building_model.building_number = tr_build_name
        community_building_model.certificate_no = tr_certificate_no
        community_building_model.regulatory_banks = tr_bank
        community_building_model.regulatory_account = tr_account
        page.close()

    # todo 更新楼栋
    def update_building(self, building_id):
        session = next(get_session())
        with session:
            statement = select(CommunityBuildingRecord).where(CommunityBuildingRecord.building_id == building_id)
            community_building_model = session.exec(statement).one()
        # 楼栋
        futures = [executor.submit(self.get_community_building_info, community_building_model)]
        fail_building_model_list = []
        for idx, future in enumerate(as_completed(futures)):
            try:
                # my_logger.info(future.result())
                is_success, result = future.result()
                my_logger.info(f"处理第{idx + 1} / {len(futures)}个任务完成")
                if is_success:
                    JinanHandler.bulk_update(CommunityBuildingRecord, result)
                else:
                    fail_building_model_list.append(result)
            except Exception as e:
                my_logger.error(f"Error processing community link task: {e}")
                my_logger.error(f"失败获取楼栋任务: {future.result()}")
        # my_logger.info(f"ctf: {len(community_data_col_building_model_list)}")
        my_logger.warning(f"失败获取楼栋任务数量: {len(fail_building_model_list)}")
        if fail_building_model_list:
            my_logger.warning(f"失败获取楼栋任务: {fail_building_model_list}")
            # 失败的 building_id = None
            JinanHandler.bulk_insert(CommunityBuildingRecord, fail_building_model_list)

    # 更新楼栋的房间, 失败就buid重跑一遍
    def update_building_room(self, building_id):
        session = next(get_session())
        futures = []
        with (session):
            statement = select(CommunityBuildingRecord).where(CommunityBuildingRecord.building_id == building_id)
            building_data = session.exec(statement).one()
            stat = select(func.count()).select_from(CommunityDataCollection).where(CommunityDataCollection.building_id == building_id)
            count = session.exec(stat).one()
            my_logger.info(f"已爬该楼栋房间数: {count}")
            # 房间数小于售房数+可售房数，则爬取该楼栋数据
            if count < building_data.sale_count + building_data.can_sale_count:
                futures.append(executor.submit(self.collect_community_table_and_room_info, building_data))
            else:
                my_logger.info(f"房间数无变化")
                return
        success_task, fail_task = JinanHandler.process_futures(futures)
        JinanHandler.bulk_insert(CommunityDataCollection, success_task)
        # 去重
        with (session):
            sql = """
                SELECT id
                FROM (
                         SELECT id,
                                ROW_NUMBER() OVER(PARTITION BY building_id ORDER BY is_house, id DESC) as row_num
                         FROM community_building_record
                     ) t
                WHERE t.row_num = 1;
            """
            session.exec(sql)
        if fail_task:
            my_logger.warning(f"失败获取room任务数量: {len(fail_task)}")
            # 失败的 room_id = None
            # JinanHandler.bulk_insert(CommunityDataCollection, fail_task)
            my_logger.warning(f"失败获取room任务: {fail_task}")

    # 更新某个房间的具体信息
    def update_room_info(self, room_id):
        pass

    @staticmethod
    def convert_unit(unit_str: str):
        if not unit_str:
            return None
        unit = unit_str.replace('单元', '')
        num = change_to_int(unit)
        if num is None:
            my_logger.error(f"单元转换失败: {unit_str}")
            raise ValueError(f"单元转换失败: {unit_str}")
        return unit

    @staticmethod
    def convert_room(room_str: str):
        if not room_str:
            return None
        room = room_str.replace('号', '')
        num = change_to_int(room)
        if num is None:
            my_logger.error(f"房间转换失败: {room_str}")
            raise ValueError(f"房间转换失败: {room_str}")
        if num < 10:
            return '0' + room
        else:
            return room

    @staticmethod
    def convert_room_number(room_number_str: str):
        if not room_number_str:
            return None
        splits = room_number_str.split('-')
        try:
            if len(splits) > 1:
                num = change_to_int(splits[1])
            elif len(splits) == 1:
                num = change_to_int(splits[0])
            else:
                my_logger.warning(f"special room: {room_number_str}")
                return None
            return num
        except Exception as e:
            my_logger.error(f"Error converting room number: {room_number_str}: {e}")
    @staticmethod
    def convert_floor(floor_str: str):
        if not floor_str:
            return None
        floor = floor_str.replace('F', '')
        num = change_to_int(floor)
        if num is None:
            my_logger.error(f"楼层转换失败: {floor_str}")
            raise ValueError(f"楼层转换失败: {floor_str}")
        return num

    @staticmethod
    def convert_total_unit(unit_str: str):
        if not unit_str:
            return 1
        num = change_to_int(unit_str)
        if num is None:
            my_logger.error(f"单元转换失败: {unit_str}")
            raise ValueError(f"单元转换失败: {unit_str}")
        return num

    @staticmethod
    def get_community_link_id(community_link: str):
        query_params = get_query_params(community_link)
        # 提取 prjno 参数
        community_id = query_params.get(JiNanUrlId.PROJECT, None)[0]
        # print(f'提取的楼栋id: {community_id}')
        return community_id
    @staticmethod
    def get_community_building_link_id(community_building_link: str):
        query_params = get_query_params(community_building_link)
        community_id = query_params.get(JiNanUrlId.BUILDING, None)[0]
        return community_id
    @staticmethod
    def get_community_table_id(community_table_link: str):
        query_params = get_query_params(community_table_link)
        community_id = query_params.get(JiNanUrlId.TABLE, None)[0]
        return community_id

    def test(self):
        session = next(get_session())
        with session:
            stat = select(func.count()).select_from(CommunityProjectRecord)
            count = session.exec(stat).one()
            print(count)
if __name__ == '__main__':
    co = ChromiumOptions().headless()
    # co = ChromiumOptions()
    # 启用多例：
    Settings.set_singleton_tab_obj(False)
    co.set_retry(2, 30)
    co.set_load_mode('eager')
    co.set_user_agent(UserAgent().random)
    b = Chromium(co)
    # JinanHandler(b, CrawlerArgs(project_type=ProjectType.HOUSE)).test()
    # JinanHandler(b, CrawlerArgs(project_type=ProjectType.HOUSE)).update_building_room('11a6b47f-aa56-477c-871a-a318b792a500')
    # JinanHandler(b, CrawlerArgs(project_type=ProjectType.HOUSE)).crawler_project()
    # JinanHandler(b, CrawlerArgs(project_type=ProjectType.HOUSE)).crawler_building()
    JinanHandler(b, CrawlerArgs(project_type=ProjectType.HOUSE)).crawler_room()
    # JinanHandler(b, CrawlerArgs(project_type=ProjectType.HOUSE)).get_house_info()
    # JinanHandler(b, CrawlerArgs(project_type=ProjectType.HOUSE)).get_house_info()
    # JinanHandler(b, CrawlerArgs(project_type=ProjectType.HOUSE)).process_community_table(JinanHandler.build_table_url_prefix + '683392e9-ee07-48b7-acd2-fc390bc4c4f2')